#!/bin/bash


#--------------------------CONFIGURATIONS-----------------------------

# Name of the executable
execroot="./../bin"
executable="cg-ns"
#executable="dg-comp"

# MPI set
use_MPI="yes"
#use_MPI="no"
MPI_impl="OpenMPI"
#MPI_impl="mpich2"



# Number of cores
num_cores=1
# Number of MPI processes (per core)
num_MPI_proc=6



# Number of OpenMP threads (per MPI process)
num_OpenMP_thr=1

# Name specifier (used to identify different experiments)
name_spec="dbg-cyl-P43"

# Input file
#infile=""
infile=${executable}.in

# Wtime
wtime="240:00:00"

#------------------------END CONFIGURATIONS---------------------------


#-----------------------------MAIN OUTPUT-----------------------------

# PBS script to submit the job
job_file="qsub_script_to_run_${executable}.sub"

# job qualifier
job_qual="${num_cores}x${num_MPI_proc}x${num_OpenMP_thr}"

# job name
job_name="${executable}_${job_qual}"

# stdout files: the first one collects the queue manager output
# (including a copy of the present file), the second one is the output
# of the executable.
job_out="out-${executable}-${name_spec}-N${job_qual}"
job_out1="${job_out}.txt"
job_out2="${job_out}-qsub.txt"

#---------------------------END MAIN OUTPUT---------------------------


# We make a copy so that the original file can be changed even while
# the job is waiting in the queue.
input_file=""
if [ -n "${infile}" ]; # notice that we need "" in the test!
 then
  input_file="temp-${job_name}-${RANDOM}"
  cp ${infile} "${input_file}"
fi

nodes_file="mpd.nodes-${RANDOM}"

# Total number of threads per core: MPI*OpenMP
num_cpus_core=`echo ${num_MPI_proc} ${num_OpenMP_thr} | awk '{ N = \$1*\$2 ; print N }'`

# Total number of MPI processes
tot_num_MPI_proc=`echo ${num_cores} ${num_MPI_proc} | awk '{ N = \$1*\$2 ; print N }'`

# Write the PBS input file
cat > ${job_file} << EOF
#!/bin/bash
#PBS -S /bin/bash

# Select the queue
#PBS -q lisa

# Allocate resources (and select maximum runtime)
#PBS -l select=${num_cores}:ncpus=${num_cpus_core}:mpiprocs=${num_MPI_proc}
#PBS -l walltime=240:00:00

# Set the job name
#PBS -N ${job_name}

# Set the output file and merge it to the sterr
#PBS -o ${job_out2}
#PBS -j oe

# Start the job in the current directory (PBS starts in the home folder)
cd \${PBS_O_WORKDIR}

# Echo the PBS input file (this file)
cat ${job_file}
echo
echo
echo "#><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>#"
echo
echo

# Preliminaries
hostname
ulimit -s unlimited # make sure we can put big arrays on the stack
ulimit -c unlimited # make sure we can produce core dumps
date

if [ "${use_MPI}" = "yes" ]
 then

if [ "${MPI_impl}" = "OpenMPI" ]
 then

# Setting up MPI variable
source /data/apps/scripts/xopenmpi_intel.sh

# Run the executable
mpiexec -np ${tot_num_MPI_proc} -machinefile \${PBS_NODEFILE}  \\
  --display-allocation -v -x OMP_NUM_THREADS=${num_OpenMP_thr} \\
  --output-filename ${job_out1}                                \\
  ${execroot}/${executable} ${input_file}

else

#---------------------------------------------------------------------#
# Now set up the mpich2 ring on the assigned processors (this follows
# section 5.8.1 of the MPICH2 User's Guide, version November 18,
# 2009).

# Setting up MPI variable
source /data/apps/scripts/xmpich2_intel.sh  

# 1) First we need to translate the file \${PBS_NODEFILE}, used by the
# PBS environment, into something that can be read by mpiexec.
cat \${PBS_NODEFILE} | uniq | \\
  awk '{ printf("%s:${num_MPI_proc}\\n", \$1); }' > ${nodes_file}

# 2) Now we set up (and test) the ring
mpdboot --verbose -f ${nodes_file} -n ${num_cores}
mpdtrace

# 3) Run the executable
mpiexec -machinefile ${nodes_file} -l -n ${tot_num_MPI_proc} \\
  -env OMP_NUM_THREADS ${num_OpenMP_thr} \\
  ${execroot}/${executable} ${input_file} > ${job_out1}

# 4) Clean up
mpdallexit
#rm ${nodes_file}

#---------------------------------------------------------------------#

fi

else

export OMP_NUM_THREADS=${num_OpenMP_thr}
${execroot}/${executable} ${input_file} > ${job_out1}

fi

date
EOF

# qsub appends the output, which can be confusing
if [ -e "${job_out2}" ]
 then
  rm ${job_out2}
fi

qsub ${job_file}

